PPG Signal Processing for the Goldilocks ppg files

In [1]:
# Initialize the package
import sys

repo = '/root/biosignal_processing_package'
sys.path.insert(1, repo)
sys.path.insert(1, repo + '/biosignal_processing_package' )

Import the package functions

In [2]:
import pandas as pd
import numpy as np

# package functions
from preprocessing.signal_transform import signal_downsample, segment_signal 
from preprocessing.signal_filter import filter_ppg
from plotting.signal_plots import plot_signal
from plotting.signal_plots import plot_filtered
from savvyppg import ppg_module
In /opt/anaconda/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: 
The text.latex.preview rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.
In /opt/anaconda/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: 
The mathtext.fallback_to_cm rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.
In /opt/anaconda/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: Support for setting the 'mathtext.fallback_to_cm' rcParam is deprecated since 3.3 and will be removed two minor releases later; use 'mathtext.fallback : 'cm' instead.
In /opt/anaconda/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: 
The validate_bool_maybe_none function was deprecated in Matplotlib 3.3 and will be removed two minor releases later.
In /opt/anaconda/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: 
The savefig.jpeg_quality rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.
In /opt/anaconda/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: 
The keymap.all_axes rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.
In /opt/anaconda/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: 
The animation.avconv_path rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.
In /opt/anaconda/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: 
The animation.avconv_args rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.
WARNING:tensorflow:From /opt/anaconda/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/compat/v2_compat.py:65: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
Instructions for updating:
non-resource variables are not supported in the long term
WARNING:tensorflow:From /opt/anaconda/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
Loading DeepBeat trained model
WARNING:tensorflow:Large dropout rate: 0.544997 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.
WARNING:tensorflow:Large dropout rate: 0.567864 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.
WARNING:tensorflow:Large dropout rate: 0.671769 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.
Done!

Reading and importing the signals

In [3]:
# read files
files = [
    "0-Potions-1622059739-ppg.csv.gz",
    "0-RelaxGame-1622059683-ppg.csv.gz",
    #"0-RelaxKaleidoscope-1620938134-ppg.csv.gz",
    "4885434061570493903-MarbleGame-1620934336-ppg.csv.gz",
    "5348144018922788108-VagalTone-1619717980-ppg.csv.gz"
]

signals = []
for file in files:
    filepath = repo + "/data/" + file
    signal = pd.read_csv(filepath, compression = 'gzip')
    signal.columns = [col.strip() for col in signal.columns] # strip away spaces from col names
    signals.append(signal)

Inspect the structure of the data and estimate the sampling rate of the signal

In [4]:
signal = signals[3]
print(signal.shape)
display(signal.head())
sampling_rate = len(signal)/(signal.ppg_time[len(signal)-2])
print("sampling rate: " + str(sampling_rate))
(22536, 5)
ppg_time ppg0 ppg1 ppg2 ambient
0 0.000000 -25379 -71592 -21379 -329832
1 0.007692 -25321 -71605 -21360 -329910
2 0.015385 -25265 -71603 -21302 -329959
3 0.023077 -25001 -71689 -21252 -329999
4 0.030769 -24955 -71631 -21147 -329938
sampling rate: 135.04562222093975
In [5]:
# comparing available signals
average_signal = (signal.ppg0+signal.ppg1+signal.ppg2)/3
plot_signal(
     [signal.ppg0.to_list(),signal.ppg1.to_list(),signal.ppg2.to_list(),average_signal.to_list()],
     [135,135,135,135], 
     labels=["ppg0","ppg1","ppg2","average"],
     x_axis_label = "Time (s)"
)
INFO:numexpr.utils:NumExpr defaulting to 8 threads.

Visualizing all the signals

In [6]:
# visualizing all sets of signals
n= 0
for signal in signals:
    print(files[n])
    plot_signal(
         [signal.ppg0.to_list(),signal.ppg1.to_list(),signal.ppg2.to_list()],
         [135,135,135], 
         labels=["ppg0","ppg1","ppg2"],
         x_axis_label = "Time (s)"
    )
    n = n + 1
0-Potions-1622059739-ppg.csv.gz
0-RelaxGame-1622059683-ppg.csv.gz
4885434061570493903-MarbleGame-1620934336-ppg.csv.gz
5348144018922788108-VagalTone-1619717980-ppg.csv.gz

Signal Processing

Signal Filtering

In [7]:
# make list of only ppg0 signals
ppg0_list = [signal.ppg0.to_list() for signal in signals]
sr = 135
# actual cleaning
ppg0_clean = [filter_ppg(ppg0_sig, sr) for ppg0_sig in ppg0_list]

Signal Cleaning Inspection

In [8]:
# comparing raw and clean signals
for i in range(len(signals)):
    print(files[i])
    plot_filtered(ppg0_list[i],ppg0_clean[i],sr)
0-Potions-1622059739-ppg.csv.gz
0-RelaxGame-1622059683-ppg.csv.gz
4885434061570493903-MarbleGame-1620934336-ppg.csv.gz
5348144018922788108-VagalTone-1619717980-ppg.csv.gz

Observation:

  • The first 3 signals, smoothing definitely happened, but signal quality still visually appears to be poor even after filtering.

  • For the 4th signal, which was the longest (175 seconds), the filtering function worked perfectly, removing wandering, motion artifacts, etc.

Testing the filter on smaller sample of ppg signal

In [9]:
# Segment the signal using `segment_signal()`
# 20 seconds
segments = segment_signal(ppg0_list[3], 135, window_time = 20, show_plot = False)

#Clean and visualize the first segment
clean_segment = filter_ppg(segments[0], sr)
plot_filtered(segments[0],clean_segment,sr)
There are segments with different window length
In [10]:
# Segment the signal using `segment_signal()`
# 10 seconds
segments = segment_signal(ppg0_list[3], 135, window_time = 10, show_plot = False)

#Clean and visualize the first segment
clean_segment = filter_ppg(segments[0], sr)
plot_filtered(segments[0],clean_segment,sr)
There are segments with different window length
In [11]:
# Segment the signal using `segment_signal()`
# 5 seconds
segments = segment_signal(ppg0_list[3], 135, window_time = 5, show_plot = False)

#Clean and visualize the first segment
clean_segment = filter_ppg(segments[0], sr)
plot_filtered(segments[0],clean_segment,sr)
There are segments with different window length

Signal Quality Inspection

In [12]:
# Preprocess and inspect beat level quality of each ppg signal
savvyppg_signals = []
for i in range(len(signals)):
    print("Processing " + files[i])
    sig = ppg_module.Signal(ppg0_list[i],sr, predict_beats=True)
    savvyppg_signals.append(sig)
Processing 0-Potions-1622059739-ppg.csv.gz
Processing 0-RelaxGame-1622059683-ppg.csv.gz
Processing 4885434061570493903-MarbleGame-1620934336-ppg.csv.gz
Processing 5348144018922788108-VagalTone-1619717980-ppg.csv.gz
In [13]:
# Visualize the beat level signal quality
from plotting.ppg_plotting import ppg_plot_quality
for i in range(len(signals)):
    print(files[i])
    ppg_plot_quality((savvyppg_signals[i]).data_ppg,(savvyppg_signals[i]).beats_df,sr)
0-Potions-1622059739-ppg.csv.gz
0-RelaxGame-1622059683-ppg.csv.gz
4885434061570493903-MarbleGame-1620934336-ppg.csv.gz
5348144018922788108-VagalTone-1619717980-ppg.csv.gz
In [ ]: